package com.g2room.driving.grap.article.kc;

import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.g2room.driving.grap.article.to.ArticlesTo;
import com.g2room.driving.util.DbUtil;
import com.g2room.driving.util.IpUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

public class GrabQqDataByProxy {
	static Connection connection = null;
	static PreparedStatement  ps = null;
	static Map<String, String> errorUrls = new HashMap<String, String>();
	private static final String soruce = "腾讯汽车";
	
	private static Connection getConn() throws ClassNotFoundException, SQLException {
		if(null == connection) {
		connection = DbUtil.getConn();
		}
		return connection;
	}
	public static void main(String[] args) throws Exception{
		GrabQqDataByProxy gkd = new GrabQqDataByProxy();
		gkd.getNav();
		// 最后输出失败的链接到数据库
		insertError2Db();
    }
	
	private WebClient getWebClient() {
		 //创建一个webclient
		WebClient wc = new WebClient();
		String ip = IpUtil.getIp();
		wc.addRequestHeader("X-Real-IP", ip);
		wc.addRequestHeader("x-forwarded-for", ip);
		wc.addRequestHeader("WL-Proxy-Client-IP", ip);
		////设置请求报文头里的User-Agent字段
        wc.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2");
        //wc.addRequestHeader和request.setAdditionalHeader功能应该是一样的。选择一个即可。
        //其他报文头字段可以根据需要添加
        wc.getCookieManager().setCookiesEnabled(false);//开启cookie管理
        wc.getOptions().setJavaScriptEnabled(false);//开启js解析。对于变态网页，这个是必须的
        wc.getOptions().setCssEnabled(false);//开启css解析。对于变态网页，这个是必须的。
        wc.getOptions().setThrowExceptionOnFailingStatusCode(false);
        wc.getOptions().setThrowExceptionOnScriptError(false);
        wc.getOptions().setTimeout(10000);
        
        //htmlunit 对css和javascript的支持不好，所以请关闭之
        wc.getOptions().setJavaScriptEnabled(false);
        wc.getOptions().setCssEnabled(false);
        return wc;
	}
	
	/**
	 * 获取头部导航菜单
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getNav() throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		Map<String, String> pages = new HashMap<String, String>();
//		pages.put("http://auto.qq.com/l/service/servicing/li.htm", "维修养护");
//		pages.put("http://auto.qq.com/l/service/policy/li.htm", "政策法规");
		pages.put("http://auto.qq.com/l/service/sharing/li.htm", "车友锦囊");
//		pages.put("http://auto.qq.com/l/service/automotive/li.htm", "用品配件");
		
        for(String key : pages.keySet()) {
	        try {
	        	Thread.sleep(IpUtil.genRandomNum() * 500);
	        } catch (InterruptedException e) {
	        	e.printStackTrace();
	        }
	        getPage(key, pages.get(key));
        }
	}
	
	/**
	 * 获取分页数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getPage(String url, String typeName) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get page data!" + typeName);
		WebClient wc = getWebClient();
		//获取页面
		HtmlPage page = null;
        try {
        	page = wc.getPage(url);
        } catch (Exception e) {
        	errorUrls.put(url, typeName);
        	e.printStackTrace();
        	return;
        }
		// 获取分页数据
		List<?> list = page.getByXPath("//div[@class=\"pageNav\"]/a");
		List<String> hrefs = new ArrayList<String>();
//		hrefs.add(url);
		HtmlAnchor htmlAnchor = (HtmlAnchor) list.get(list.size() - 2);
		int pageNum = Integer.valueOf(htmlAnchor.asText());
		String baseUrl = htmlAnchor.getAttribute("href").replace(pageNum+".htm", "###.htm");
        if (null != wc) wc.close();
        
        for(int i = 19; i<=pageNum; i++) {
        	hrefs.add(baseUrl.replace("###.htm", i+".htm"));
        }
        
        for(String href : hrefs) {
        	try {
				Thread.sleep(IpUtil.genRandomNum() * 500);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
			getList(href, typeName);
        }
	}
	
	/**
	 * 获取列表数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getList(String url, String typeName) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get list data!url==  " + url + "   ,typeName=" + typeName);
		WebClient wc = getWebClient();
		//获取页面
		HtmlPage page = null;
        try {
        	page = wc.getPage(url);
        } catch (Exception e) {
        	errorUrls.put(url, typeName);
        	e.printStackTrace();
        	return;
        }
		// 获取分页数据
		List<?> list = page.getByXPath("//div[@class=\"mod newslist\"]/ul/li/a");
		List<ArticlesTo> articles = new ArrayList<ArticlesTo>();
		for(Object hs : list) {
			HtmlAnchor a = (HtmlAnchor) hs;
			ArticlesTo articlesTo = new ArticlesTo();
			articlesTo.setTypeName(typeName);
			articlesTo.setTitle(a.asText());
			articlesTo.setSourceUrl(a.getAttribute("href"));
			articles.add(articlesTo);
		}
        if (null != wc) wc.close();
        
        for(ArticlesTo articlesTo : articles) {
        	try {
        		Thread.sleep(IpUtil.genRandomNum() * 500);
        	} catch (InterruptedException e) {
        		e.printStackTrace();
        	}
        	getDetail(articlesTo);
        }
	}
	
	public void getDetail(ArticlesTo articles) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get detail data!  " + articles.getSourceUrl());
		WebClient wc = getWebClient();
		HtmlPage page = null;
        try {
        	page = wc.getPage(articles.getSourceUrl());
        } catch (Exception e) {
        	errorUrls.put(articles.getSourceUrl(), articles.getTypeName());
        	e.printStackTrace();
        	return;
        }
        
		DomElement div = page.getElementById("Cnt-Main-Article-QQ");
		if(null != div) {
			articles.setContent(div.asXml());
			insert2Db(articles);
		}
        if (null != wc) wc.close();
	}
	int sqlCount = 0;
	public void insert2Db(ArticlesTo articles) {
		try {
			if(sqlCount == 0) ps = getConn().prepareStatement("insert into d_article_grap(title,content,publish_time,type_name,source,source_url,state) values(?,?,?,?,?,?,2)");
			// title,content,publish_time,type_name,source,source_url
			ps.setString(1, articles.getTitle());
			ps.setString(2, articles.getContent());
			ps.setString(3, articles.getPublishTime());
			ps.setString(4, articles.getTypeName());
			ps.setString(5, soruce);
			ps.setString(6, articles.getSourceUrl());
			ps.execute();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	
	public static void insertError2Db() {
		try {
			
			PreparedStatement  ps = getConn().prepareStatement("insert into d_article_grap_err(title,source,source_url,type_name) values(?,?,?,?)");
			for(String url : errorUrls.keySet()) {
				ps.setString(1, errorUrls.get(url));
				ps.setString(2, soruce);
				ps.setString(3, url);
				ps.setString(4, "导航和列表错误");
				ps.addBatch();
			}
//			for(ArticlesTo article : articleList) {
//				ps.setString(1, article.getTitle());
//				ps.setString(2, soruce);
//				ps.setString(3, article.getSourceUrl());
//				ps.setString(4, article.getTypeName());
//				ps.addBatch();
//			}
			ps.executeBatch();
			if(null != ps) ps.close();
			if(null != connection) connection.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
}
